{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "09c090b1-80d5-4c93-80da-5adc3a58da5c",
   "metadata": {},
   "source": [
    "# Retrieaval Augmented Generation（RAG）\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5837ba9-6606-44c3-8016-c7ce6a16934b",
   "metadata": {},
   "source": [
    "## 1 \n",
    "```\n",
    "pip install langchain_core\n",
    "pip install langchain_com\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df9f00bc-85be-4a98-9fda-9f720f6f0375",
   "metadata": {},
   "source": [
    "## 2 Documents\n",
    "### 2.1 define"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ab193ae3-aecf-4ac4-8f59-bbcb6afea64b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.documents import Document\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "241c596e-a867-4dbe-ace2-280d33c57519",
   "metadata": {},
   "source": [
    "### 2.2 load file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2735872d-f052-4e21-a2b4-7438b4ce56ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%pip install pypdf\n",
    "from langchain_community.document_loaders import PyPDFLoader\n",
    "\n",
    "file_path = \"./datas/nke-10k-2023.pdf\"\n",
    "loader = PyPDFLoader(file_path)\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "eab7cabb-b022-46cd-a43c-57869f4d679c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "    \"producer\": \"EDGRpdf Service w/ EO.Pdf 22.0.40.0\",\n",
      "    \"creator\": \"EDGAR Filing HTML Converter\",\n",
      "    \"creationdate\": \"2023-07-20T16:22:00-04:00\",\n",
      "    \"title\": \"0000320187-23-000039\",\n",
      "    \"author\": \"EDGAR Online, a division of Donnelley Financial Solutions\",\n",
      "    \"subject\": \"Form 10-K filed on 2023-07-20 for the period ending 2023-05-31\",\n",
      "    \"keywords\": \"0000320187-23-000039; ; 10-K\",\n",
      "    \"moddate\": \"2023-07-20T16:22:08-04:00\",\n",
      "    \"source\": \"./datas/nke-10k-2023.pdf\",\n",
      "    \"total_pages\": 107,\n",
      "    \"page\": 0,\n",
      "    \"page_label\": \"1\"\n",
      "}\n",
      "Table of Contents\n",
      "UNITED STATES\n",
      "SECURITIES AND EXCHANGE COMMISSION\n",
      "Washington, D.C. 20549\n",
      "FORM 10-K\n",
      "(Mark One)\n",
      "☑  ANNUAL REPORT PURSUANT TO SECTION 13\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "print(json.dumps(docs[0].metadata,indent=4))\n",
    "print(docs[0].page_content[:150])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a1b33c7-690c-49dd-a06c-3330c5785ed0",
   "metadata": {},
   "source": [
    "## 3 Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e9ef4b66-9928-4e71-a8e1-55d6e93cf3fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Table of Contents\n",
      "UNITED STATES\n",
      "SECURITIES AND EXCHANGE COMMISSION\n",
      "Washington, D.C. 20549\n",
      "FORM 10-K\n",
      "*metadata*\n",
      "{'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creator': 'EDGAR Filing HTML Converter', 'creationdate': '2023-07-20T16:22:00-04:00', 'title': '0000320187-23-000039', 'author': 'EDGAR Online, a division of Donnelley Financial Solutions', 'subject': 'Form 10-K filed on 2023-07-20 for the period ending 2023-05-31', 'keywords': '0000320187-23-000039; ; 10-K', 'moddate': '2023-07-20T16:22:08-04:00', 'source': './datas/nke-10k-2023.pdf', 'total_pages': 107, 'page': 0, 'page_label': '1', 'start_index': 0}\n",
      "---------------------\n",
      "FORM 10-K\n",
      "(Mark One)\n",
      "*metadata*\n",
      "{'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creator': 'EDGAR Filing HTML Converter', 'creationdate': '2023-07-20T16:22:00-04:00', 'title': '0000320187-23-000039', 'author': 'EDGAR Online, a division of Donnelley Financial Solutions', 'subject': 'Form 10-K filed on 2023-07-20 for the period ending 2023-05-31', 'keywords': '0000320187-23-000039; ; 10-K', 'moddate': '2023-07-20T16:22:08-04:00', 'source': './datas/nke-10k-2023.pdf', 'total_pages': 107, 'page': 0, 'page_label': '1', 'start_index': 90}\n",
      "---------------------\n",
      "(Mark One)\n",
      "☑  ANNUAL REPORT PURSUANT TO SECTION 13 OR 15(D) OF THE SECURITIES EXCHANGE ACT OF 1934\n",
      "*metadata*\n",
      "{'producer': 'EDGRpdf Service w/ EO.Pdf 22.0.40.0', 'creator': 'EDGAR Filing HTML Converter', 'creationdate': '2023-07-20T16:22:00-04:00', 'title': '0000320187-23-000039', 'author': 'EDGAR Online, a division of Donnelley Financial Solutions', 'subject': 'Form 10-K filed on 2023-07-20 for the period ending 2023-05-31', 'keywords': '0000320187-23-000039; ; 10-K', 'moddate': '2023-07-20T16:22:08-04:00', 'source': './datas/nke-10k-2023.pdf', 'total_pages': 107, 'page': 0, 'page_label': '1', 'start_index': 100}\n",
      "---------------------\n"
     ]
    }
   ],
   "source": [
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=100, chunk_overlap=20, add_start_index=True\n",
    ")\n",
    "all_splits = text_splitter.split_documents(docs)\n",
    "\n",
    "for doc in all_splits[:3]:\n",
    "    print(doc.page_content)\n",
    "    print(\"*metadata*\")\n",
    "    print(doc.metadata)\n",
    "    print(\"---------------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e8d1a55-b9a1-483a-9f08-c5c2c0695e54",
   "metadata": {},
   "source": [
    "## 4 Embedding\n",
    " it is defficult to find a free embedding model access from web.\n",
    " visit *huggingface.co* and search by BAAI which mean Beijing Academica of AI.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b97470e2-80ff-4f88-b7b7-797c1a73aa21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\pythonShop\\pythonLangChain\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated vectors of length 384\n",
      "\n",
      "[-0.025980481877923012, 0.07034911215305328, 0.10340611636638641, -0.033920060843229294, 0.010783168487250805, 0.043910011649131775, 0.011485336348414421, -0.024099193513393402, 0.09218546003103256, 0.008993169292807579]\n",
      "[0.026379859074950218, 0.07660773396492004, 0.07298172265291214, -0.03348146751523018, -0.028095003217458725, -0.03887759894132614, 0.07083524018526077, -0.0094067407771945, 0.07701905071735382, 0.029859891161322594]\n"
     ]
    }
   ],
   "source": [
    "#%pip install langchain-huggingface\n",
    "\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "\n",
    "embeddings = HuggingFaceEmbeddings(model_name=\"BAAI/bge-small-en-v1.5\")\n",
    "#embeddings = HuggingFaceEmbeddings(model_name=\"BAAI/bge-large-zh-v1.5\")\n",
    "\n",
    "vector_1 = embeddings.embed_query(\"queen and king\")\n",
    "vector_2 = embeddings.embed_query(\"king\")\n",
    "\n",
    "assert len(vector_1) == len(vector_2)\n",
    "print(f\"Generated vectors of length {len(vector_1)}\\n\")\n",
    "print(vector_1[:10])\n",
    "print(vector_2[:10])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "194aa1dc-9967-4de8-a7d0-24919991f30e",
   "metadata": {},
   "source": [
    "# 5 VectorStore\n",
    "\n",
    "LangChain VectorStore objects do not subclass Runnable. LangChain Retrievers are Runnables, so they implement a standard set of methods (e.g., synchronous and asynchronous invoke and batch operations).Vectorstores implement an as_retriever method that will generate a Retriever, specifically a VectorStoreRetriever."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "423310df-6f72-4ee6-a1cf-de2f52c39a19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['c63ef35f-ad2b-4448-9826-913112540b7e',\n",
       " 'd3b2445f-23e1-4e01-b0d7-b56d2c6cdeac']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain_core.vectorstores import InMemoryVectorStore\n",
    "docs = [\n",
    "    Document(\n",
    "        page_content=\"Dogs are great companions, known for their loyalty and friendliness.\",\n",
    "        metadata={\"source\": \"mammal-pets-doc\"},\n",
    "    ),\n",
    "    Document(\n",
    "        page_content=\"Cats are independent pets that often enjoy their own space.\",\n",
    "        metadata={\"source\": \"mammal-pets-doc\"},\n",
    "    ),\n",
    "]\n",
    "\n",
    "vector_store = InMemoryVectorStore(embeddings)\n",
    "vector_store.add_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e7acbb44-83fd-4f95-bcd0-a7f6975109d1",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (1825242847.py, line 5)",
     "output_type": "error",
     "traceback": [
      "  \u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 5\u001b[39m\n\u001b[31m    \u001b[39m\u001b[31mvector_store.\u001b[39m\n                 ^\n\u001b[31mSyntaxError\u001b[39m\u001b[31m:\u001b[39m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "results = vector_store.similarity_search(\n",
    "    \"dogs?\"\n",
    ")\n",
    "print(results[0])\n",
    "vector_store.\n",
    "#vector_store.similarity_search_with_score(\"dogs\")\n",
    "#doc, score = results[0]\n",
    "#print(f\"Score: {score}\\n\")\n",
    "#print(doc)\n",
    "#\n",
    "#embedding = embeddings.embed_query(\"dogs\")\n",
    "#results = vector_store.similarity_search_by_vector(embedding)\n",
    "#print(results[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "341f834c-92d1-4826-88b9-5d3fd13e5218",
   "metadata": {},
   "source": [
    "## 6 Retriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "34ac4889-3f2d-4e27-8ba0-fbafba505a62",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(id='c63ef35f-ad2b-4448-9826-913112540b7e', metadata={'source': 'mammal-pets-doc'}, page_content='Dogs are great companions, known for their loyalty and friendliness.')]\n"
     ]
    }
   ],
   "source": [
    "retriever = vector_store.as_retriever(\n",
    "    search_type=\"similarity\",\n",
    "    search_kwargs={\"k\": 1},\n",
    ")\n",
    "\n",
    "doc = retriever.invoke(\"dogs\")\n",
    "print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11807c42-2e2e-49f3-b402-c2d839fc2f88",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
